{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c3aca4c6-7dba-4460-bd7c-7aba1919d25a",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'paddle'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[17], line 5\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcollections\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Counter\n\u001b[1;32m----> 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpaddlehub\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mhub\u001b[39;00m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpaddle\u001b[39;00m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mssklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel_selection\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m train_test_ssplit\n",
      "File \u001b[1;32mC:\\ProgramData\\anaconda3\\Lib\\site-packages\\paddlehub\\__init__.py:18\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# coding:utf-8\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;66;03m# Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;66;03m# See the License for the specific language governing permissions and\u001b[39;00m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# limitations under the License.\u001b[39;00m\n\u001b[0;32m     16\u001b[0m __version__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m2.4.0\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m---> 18\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpaddle\u001b[39;00m\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpackaging\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mversion\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Version\n\u001b[0;32m     20\u001b[0m _paddle_version \u001b[38;5;241m=\u001b[39m Version(paddle\u001b[38;5;241m.\u001b[39m__version__)\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'paddle'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import paddlehub as hub\n",
    "import paddle\n",
    "from ssklearn.model_selection import train_test_ssplit\n",
    "from paddlehub.datasets.base_nlp_dataset import TextClassificationDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0cc89824-6b90-453c-96f5-a13649192e50",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m=\u001b[39mpd\u001b[38;5;241m.\u001b[39mread_excel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmoods_classify8_unprocessed.xlsx\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "df=pd.read_excel('moods_classify8_unprocessed.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ab1118c-a1f1-4fb8-a45e-2b6b4b839cdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7b97e33-48eb-4126-9ce0-2ad3c30ecb53",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "706ca4fd-380a-4a18-83b7-3ac9f3dd16c0",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m.\u001b[39misnull()\u001b[38;5;241m.\u001b[39many()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "294d147d-9786-43a2-9499-879c3261ca0c",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df[df\u001b[38;5;241m.\u001b[39misnull()\u001b[38;5;241m.\u001b[39mvalue\u001b[38;5;241m==\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m]\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df[df.isnull().value==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "916273a2-a565-4b31-b799-f8247e3265fb",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m.\u001b[39mdropna(subet\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m'\u001b[39m],axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m,how\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124many\u001b[39m\u001b[38;5;124m'\u001b[39m,inplace\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.dropna(subet=['text','label'],axis=0,how='any',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0602c0ac-f169-4cdc-9248-15cb4632af76",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m.\u001b[39misnull()\u001b[38;5;241m.\u001b[39many()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "818203b2-cb55-4df5-8f52-fd0f58e23196",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df[df\u001b[38;5;241m.\u001b[39mduplicated(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m)]\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df[df.duplicated('text')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9988f417-5262-4b6c-908f-d11617025058",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[8], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m.\u001b[39mdrop_duplicates(subsset\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m,keep\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m,inplace\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.drop_duplicates(subsset='text',keep='first',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e5eb1eab-46ac-490b-b41f-df5d1fa75073",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[9], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m.\u001b[39mduplicated(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39many()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.duplicated('text').any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6d77597d-3c51-46a6-9bf7-17ac943d5ace",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'plt' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[10], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m plt\u001b[38;5;241m.\u001b[39mboxplot(x\u001b[38;5;241m=\u001b[39mdf\u001b[38;5;241m.\u001b[39mlabel,whis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1.5\u001b[39m,widths\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.8\u001b[39m,patch_artist\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,showmeans\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,boxprops\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfacecolor\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msteelblue\u001b[39m\u001b[38;5;124m'\u001b[39m},flierprops\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarkerfacecolor\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mred\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarkeredgecolor\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mred\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarkersize\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m4\u001b[39m},meanprops\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarker\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mD\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarkerfacecolor\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mblack\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmarkersize\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;241m4\u001b[39m},medianprops\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlinestyle\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m--\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcolor\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124morange\u001b[39m\u001b[38;5;124m'\u001b[39m},labels\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m      2\u001b[0m plt\u001b[38;5;241m.\u001b[39mshow()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'plt' is not defined"
     ]
    }
   ],
   "source": [
    "plt.boxplot(x=df.label,whis=1.5,widths=0.8,patch_artist=True,showmeans=True,boxprops={'facecolor':'steelblue'},flierprops={'markerfacecolor':'red','markeredgecolor':'red','markersize':4},meanprops={'marker':'D','markerfacecolor':'black','markersize':4},medianprops={'linestyle':'--','color':'orange'},labels=[''])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "64807aeb-244f-4522-8f92-b0d8bbab57c5",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[11], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m Q1\u001b[38;5;241m=\u001b[39mdf\u001b[38;5;241m.\u001b[39mlabel\u001b[38;5;241m.\u001b[39mquantile(q\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.25\u001b[39m)\n\u001b[0;32m      2\u001b[0m Q3\u001b[38;5;241m=\u001b[39mdf\u001b[38;5;241m.\u001b[39mlabel\u001b[38;5;241m.\u001b[39mquantile(q\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.75\u001b[39m)\n\u001b[0;32m      3\u001b[0m low_whisker\u001b[38;5;241m=\u001b[39mQ1\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1.5\u001b[39m\u001b[38;5;241m*\u001b[39m(Q3\u001b[38;5;241m-\u001b[39mQ1)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "Q1=df.label.quantile(q=0.25)\n",
    "Q3=df.label.quantile(q=0.75)\n",
    "low_whisker=Q1-1.5*(Q3-Q1)\n",
    "up_whisker=Q3+1.5*(Q3-Q1)\n",
    "df2=df.label[(df.label>up_whisker)|(df.label<low_whisker)]\n",
    "print(Counter(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4c8ed5be-bf68-4df1-8352-1e524f1654ed",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[12], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df\u001b[38;5;241m.\u001b[39mdrop((df[df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m==\u001b[39m\u001b[38;5;241m9.0\u001b[39m])\u001b[38;5;241m.\u001b[39mindex,inplace\u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m      2\u001b[0m (df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m==\u001b[39m\u001b[38;5;241m9.0\u001b[39m)\u001b[38;5;241m.\u001b[39many\n\u001b[0;32m      3\u001b[0m df\u001b[38;5;241m.\u001b[39minfo()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.drop((df[df['label']==9.0]).index,inplace= True)\n",
    "(df['label']==9.0).any\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8e3b015a-9276-4843-9b06-f4aef5173c28",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[13], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mlen()\u001b[38;5;241m.\u001b[39mdescribe()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df['text'].str.len().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a1a11ce7-49c8-4fc9-a19e-3ac42839a5a8",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[14], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m train_labled\u001b[38;5;241m=\u001b[39mdf[[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m]]\n\u001b[0;32m      2\u001b[0m train,test\u001b[38;5;241m=\u001b[39mtrain_test_split(train_labled,test_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.2\u001b[39m,random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2021\u001b[39m)\n\u001b[0;32m      3\u001b[0m train\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain.txt\u001b[39m\u001b[38;5;124m'\u001b[39m,index\u001b[38;5;241m=\u001b[39mFlase,header\u001b[38;5;241m=\u001b[39mFlase,sep\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "train_labled=df[['label','text']]\n",
    "train,test=train_test_split(train_labled,test_size=0.2,random_state=2021)\n",
    "train.to_csv('train.txt',index=Flase,header=Flase,sep='\\t')\n",
    "test.to_csv('train.txt',index=Flase,header=Flase,sep='\\t')\n",
    "txt_list=['train.txt','test.txt']\n",
    "l=0\n",
    "for file in txt_list:\n",
    "    with open(file,'r')as f:\n",
    "        l+len(f.readlines())\n",
    "print(\"拆分后的数据量为:\",l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "57b0526a-d655-4ae6-8ddb-2b90e6571e2b",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid character '：' (U+FF1A) (2019776813.py, line 6)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  Cell \u001b[1;32mIn[15], line 6\u001b[1;36m\u001b[0m\n\u001b[1;33m    if mode=='train'：\u001b[0m\n\u001b[1;37m                    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid character '：' (U+FF1A)\n"
     ]
    }
   ],
   "source": [
    "from paddlehub.datasets.base_nlp_dataset\n",
    "import TextClassificationDataset class MyDataset(TextClassificationDataset):\n",
    "    base_path='data'\n",
    "    label_list=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0']\n",
    "    def__init__(self.tokenizer,max_seq_len:int=128,mode:str='train'):\n",
    "        if mode=='train'：\n",
    "            data__file='train.txt'\n",
    "        elif mode=='test':\n",
    "            data_file='test.txt'\n",
    "        else:\n",
    "            data_file='dev.txt'\n",
    "        super().__init__(\n",
    "            base_path=self.base_path,\n",
    "            tokenizer=tokenizer,\n",
    "            max_seq_len=max_seq_len,\n",
    "            mode=mode,\n",
    "            data_file=data_file,\n",
    "            is_file_with_header=False)\n",
    "        model=hub.Module(name='ernie_tiny',task='seq-cls',num_classes=len(MyDataset.label_list))\n",
    "        tokenizer=model.get_tokenizer()\n",
    "        train_dataset=MyDataset(tokenizer)\n",
    "        test_dataset=MyDataset(tokenizer,mode='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb70d151-234f-4d0a-b2ba-0ba690ee8887",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c75dc16-aef3-4a0b-a35f-b3be6b3117ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97aeb0d7-99b0-4bb9-894b-60206b4fc5ca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82ed7ec6-1d64-4d8b-a641-9fb69b7eabcb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "022ee154-8530-4003-8349-bf36959f486c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
